{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f0335ed4-7f0e-4402-ba61-b6377511135e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "def mean_squared_error(y,t):\n",
    "    '''均方误差'''\n",
    "    return 0.5 * np.sum((y - t)**2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3229e9b5-eb65-4f8c-826a-976bc18fe5ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.09750000000000003"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = [0, 0, 1, 0, 0, 0, 0, 0, 0, 0]\n",
    "y = [0.1, 0.05, 0.6, 0.0, 0.05, 0.1, 0.0, 0.1, 0.0, 0.0]\n",
    "mean_squared_error(np.array(y), np.array(t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6611bafd-4271-4de5-ba7b-8fdb888bb80c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5975"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = [0.1, 0.05, 0.1, 0.0, 0.05, 0.1, 0.0, 0.6, 0.0, 0.0]\n",
    "mean_squared_error(np.array(y), np.array(t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c923c3c0-f80b-4cda-b6f6-7c409ff24019",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy_error(y, t):\n",
    "    '''交叉熵误差'''\n",
    "    delta = 1e-7\n",
    "    return -np.sum(t * np.log(y + delta))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ca489c97-1072-4c33-9586-95c884432cf0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.510825457099338"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = [0, 0, 1, 0, 0, 0, 0, 0, 0, 0]\n",
    "y = [0.1, 0.05, 0.6, 0.0, 0.05, 0.1, 0.0, 0.1, 0.0, 0.0]\n",
    "cross_entropy_error(np.array(y), np.array(t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "da91f081-6474-41a3-ae32-2b5c85a6f38e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.302584092994546"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = [0.1, 0.05, 0.1, 0.0, 0.05, 0.1, 0.0, 0.6, 0.0, 0.0]\n",
    "cross_entropy_error(np.array(y), np.array(t))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "715bd726-5604-4e64-bd80-01dd6e7434e7",
   "metadata": {},
   "source": [
    "## mini-batch学习\n",
    "MNIST 数据集的训练数据有 60000 个，如果以全部数据为对象\n",
    "求损失函数的和，则计算过程需要花费较长的时间。再者，如果遇到大数据，\n",
    "数据量会有几百万、几千万之多，这种情况下以全部数据为对象计算损失函\n",
    "数是不现实的。因此，我们从全部数据中选出一部分，作为全部数据的“近\n",
    "似”。神经网络的学习也是从训练数据中选出一批数据（称为 mini-batch, 小\n",
    "批量），然后对每个 mini-batch 进行学习。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "810e8cf2-a715-41a0-82e3-1ac8f75e3fac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading train-images-idx3-ubyte.gz ... \n",
      "Done\n",
      "Downloading train-labels-idx1-ubyte.gz ... \n",
      "Done\n",
      "Downloading t10k-images-idx3-ubyte.gz ... \n",
      "Done\n",
      "Downloading t10k-labels-idx1-ubyte.gz ... \n",
      "Done\n",
      "Converting train-images-idx3-ubyte.gz to NumPy Array ...\n",
      "Done\n",
      "Converting train-labels-idx1-ubyte.gz to NumPy Array ...\n",
      "Done\n",
      "Converting t10k-images-idx3-ubyte.gz to NumPy Array ...\n",
      "Done\n",
      "Converting t10k-labels-idx1-ubyte.gz to NumPy Array ...\n",
      "Done\n",
      "Creating pickle file ...\n",
      "Done!\n",
      "(60000, 784)\n",
      "(60000, 10)\n"
     ]
    }
   ],
   "source": [
    "import os, sys\n",
    "sys.path.append(os.pardir)\n",
    "import numpy as np\n",
    "from dataset.mnist import load_mnist\n",
    "\n",
    "(x_train, t_train), (x_test, t_test) = load_mnist(normalize=True, one_hot_label=True)\n",
    "\n",
    "print(x_train.shape) # (60000, 784)\n",
    "print(t_train.shape) # (60000, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d485c535-c888-4c46-b645-5c1b2ae41384",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_size = x_train.shape[0]\n",
    "batch_size = 10\n",
    "batch_mask = np.random.choice(train_size, batch_size)\n",
    "x_batch = x_train[batch_mask]\n",
    "t_batch = t_train[batch_mask]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b692efb8-1f42-415c-a9e9-96ef8920340b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([42936, 20063, 58264, 29398, 43227, 44262, 47758, 50906,  6458,\n",
       "       23546])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.choice(60000, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "621820c9-f28e-4990-90a8-908477948c13",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy_error(y, t):\n",
    "    '''带mini-batch的交叉熵误差'''\n",
    "    if y.ndim == 1:\n",
    "        t = t.reshape(1, t.size)\n",
    "        y = y.reshape(1, y.size)\n",
    "        \n",
    "    batch_size = y.shape[0]\n",
    "    return -np.sum(t * np.log(y + 1e-7)) / batch_size"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65ec3051-da06-48f8-8738-5afae679f5bb",
   "metadata": {},
   "source": [
    "## 梯度法\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f7376cfb-10b8-4421-9815-c8ffb64ba70b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def numerical_gradient(f, x):\n",
    "    h = 1e-4\n",
    "    grad = np.zeros_like(x)\n",
    "    for idx in range(x.size):\n",
    "        tmp_val = x[idx]\n",
    "        x[idx] = tmp_val + h\n",
    "        fxh1 = f(x)\n",
    "        \n",
    "        x[idx] = tmp_val - h\n",
    "        fxh2 = f(x)\n",
    "        grad[idx] = (fxh1 - fxh2) / (2 * h)\n",
    "        x[idx] = tmp_val\n",
    "    return grad\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cbb15d7e-6ced-48b0-8f7d-e4fd8fa3e68a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def function_2(x):\n",
    "    return x[0]**2 + x[1]**2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b6b2b5bd-a9c7-45dc-97e7-2336e79e524d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([6., 8.])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numerical_gradient(function_2, np.array([3.0, 4.0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "6adda0af-87f3-4563-ad41-1331cd8cc183",
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient_descent(f, init_x, lr=0.01, step_num = 100):\n",
    "    x = init_x\n",
    "    for i in range(step_num):\n",
    "        grad = numerical_gradient(f, x)\n",
    "        x -= lr * grad\n",
    "    return x"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ae3bfc2-b294-46e3-b38f-cf7f3ebcef74",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 问题\n",
    "请使用梯度法求f(x0, x1) = x0 ^ 2 + x1 ^ 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ec50087f-280d-4e51-826c-5087cb93cb13",
   "metadata": {},
   "outputs": [],
   "source": [
    "def function_2(x):\n",
    "    return np.sum(x**2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "8f1ab650-43a8-4947-90ab-cb39dddc0228",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-6.11110793e-10,  8.14814391e-10])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "init_x = np.array([-3.0, 4.0])\n",
    "gradient_descent(function_2, init_x = init_x, lr = 0.1, step_num = 100)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "7dc5414f-4955-4002-bbc5-0dbc67c7456c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-2.58983747e+13, -1.29524862e+12])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 学习率过大\n",
    "init_x = np.array([-3.0, 4.0])\n",
    "gradient_descent(function_2, init_x = init_x, lr = 10, step_num = 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "8cd9eea9-5237-4a74-841c-61ac0b17dfb5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-2.99999994,  3.99999992])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 学习率过小\n",
    "init_x = np.array([-3.0, 4.0])\n",
    "gradient_descent(function_2, init_x = init_x, lr = 1e-10, step_num = 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35b9a534-615c-48db-aa12-1bf7840d64d8",
   "metadata": {},
   "source": [
    "## 神经网络的梯度\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7bf8af4-96d8-4ed8-8ebf-2eab51f0610f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys, os\n",
    "import numpy as np\n",
    "from common.functions import softmax, cross_entropy_error\n",
    "from common.gradient import numerical_gradient\n",
    "\n",
    "class simpleNet:\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.W = np.random.randn(2, 3)\n",
    "    \n",
    "    def predict(self, x):\n",
    "        return np.dot(x, self.W)\n",
    "    \n",
    "    def loss(self, x, t):\n",
    "        z = self.predict(x)\n",
    "        y = softmax(z)\n",
    "        loss = cross_entropy_error(y,t)\n",
    "        \n",
    "        return loss\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
